package com.coderfans.pageprocessor;

import com.coderfans.datagithub.ConsolePipeline;
import com.coderfans.jedistest.RedisCacheClientTest;
import com.coderfans.redis.RedisConfig;
import com.coderfans.redis.impl.RedisCacheClient;
import redis.clients.jedis.JedisPool;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;


/**
 * Created by ${fanchunshuai} on 2016-10-12.
 *
 * @version: V1.0
 * @Desc: 爬取github的所有链接到redis中
 * 该方法在爬取初期的时候效果非常明显，但是后期爬取效率变低
 * 由于内存等原因导致内存溢出，重启则费时费力，建议使用
 * LinkStoreRedisPageTest
 * @Copyright (c) 2016 58到家-版权所有
 */
public class LinkStoreRedisTEST implements PageProcessor {
    //步骤一：抓取网站的相关配置 ： 包括编码 抓取间隔 重试次数等
    private Site site = Site.me().setRetryTimes(3).setSleepTime(1000);
    private JedisPool jedisPool = RedisConfig.generatJedisPool();
    private RedisCacheClient setRedis = new RedisCacheClient(jedisPool);

    @Override
    public void process(Page page) {

        page.addTargetRequests(page.getHtml().links().regex("(https://github\\.com/\\w+/\\w+)").all());
        page.addTargetRequests(page.getHtml().links().regex("(https://github\\.com/\\w+)").all());
        long size = setRedis.sadd(RedisKeyTest2.GITHUB_URL_TEST,"_TEST",page.getUrl().get());
        System.out.println("size = "+size);
        System.out.println("url188888881 = "+page.getUrl().get());
        //page.putField("readme", page.getHtml().xpath("//div[@id='readme']/tidyText()"));

    }

    public Site getSite() {
        return site;
    }

    public static void main(String[] args) {
        Spider.create(new LinkStoreRedisTEST()).addPipeline(new ConsolePipeline())
                .addUrl("https://github.com/search?o=asc&p=2&q=storm&s=stars&type=Repositories&utf8=%E2%9C%93")
                .thread(8).run();

    }
}
